<?php
require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';

class Zend_Search_Lucene_Analysis_Analyzer_Common_Myanlalysis extends Zend_Search_Lucene_Analysis_Analyzer_Common { 
     
    private $_position=0; 
     
    private $_cnStopWords = array(); 
    
    private $_thisAllChineseArray=array();
    
    private $_sumTime=0;

     
    public function setCnStopWords($cnStopWords){ 
        $this->_cnStopWords = $cnStopWords; 
    } 

    /** 
     * Reset token stream 
     */ 
    public function reset() 
    { 
        $this->_position = 0; 
        //$this->_input=preg_replace("/[^\x80-\xff]/",'',$this->_input);
        $this->_input=preg_replace("/[^\xe0-\xef\x80-\xbf]/",'',$this->_input);
      	$search = array(",", "/", "\\", ".", ";", ":", '"', "!", "~", "`", "^", "(", ")", "?", "-", "t", "n", "'", "<", ">", "r", "rn", "$", "&", "%", "#", "@", "+", "=", "{", "}", "[", "]", "：", "）", "（", "．", "。", "，", "！", "；", "“", "”", "‘", "’", "［", "］", "、", "—", "　", "《", "》", "－", "…", "【", "】",);
        $this->_input = str_replace($search,'',$this->_input); 
        $this->_input = str_replace($this->_cnStopWords,'',$this->_input); 

        $this->getChineseWords();
    } 

    /** 
     * Tokenization stream API 
     * Get next token 
     * Returns null at the end of stream 
     * 
     * @return Zend_Search_Lucene_Analysis_Token|null 
     */ 
    
    
    //二元分词
    public function nextToken3() 
    { 
        if ($this->_input === null) { 
            return null; 
        } 
        while ($this->_position < strlen($this->_input)) { 
            while ($this->_position < strlen($this->_input) && 
                    $this->_input[$this->_position]==' ' ) { 
                $this->_position++; 
            } 
            $termStartPosition = $this->_position;       
            $temp_char = $this->_input[$this->_position]; 
            $isCnWord = false; 
            if(ord($temp_char)>127){   
                $i = 0;        
                while ($this->_position < strlen($this->_input) && 
                ord( $this->_input[$this->_position] )>127) { 
                    $this->_position = $this->_position + 3; 
                    $i ++; 
                    if($i==2){ 
                        $isCnWord = true; 
                        break; 
                    } 
                } 
                if($i==1)continue; 
            }else{ 
                while ($this->_position < strlen($this->_input) && 
                ctype_alnum( $this->_input[$this->_position] )) { 
                    $this->_position++; 
                } 
            } 
            if ($this->_position == $termStartPosition) { 
                return null; 
            } 
            

            $token = new Zend_Search_Lucene_Analysis_Token( 
                                      substr($this->_input, 
                                             $termStartPosition, 
                                             $this->_position - $termStartPosition), 
                                      $termStartPosition, 
                                      $this->_position); 
                                      
            $token = $this->normalize($token); 
            if($isCnWord)$this->_position = $this->_position - 3; 
            if ($token !== null) { 
				//var_dump($token);
                return $token; 
            } 
        } 
        return null; 
    } 
    
    
    //单字分词
    public function nextToken4(){
    	
    	$first=array_sum(explode(' ',microtime()));
    	
    	if ($this->_input === null) { 
    		
            return null;
             
        }

    	$counts=strlen($this->_input)/3;

    	if($this->_position==$counts){
    		
    		return null; 
    		
    	}
	        
        $this->_position++;
    		
		$token = new Zend_Search_Lucene_Analysis_Token( 
                                      substr($this->_input,($this->_position-1)*3,3), 
                                      $this->_position-1, 
                                      $this->_position);

        $token = $this->normalize($token);

        if ($token !== null) {
        	
			$last=array_sum(explode(' ',microtime()));
                
            $this->_sumTime+=$last-$first;
                
           	echo $this->_sumTime.'<br />';
        	
        	var_dump($token);
            
            return $token;
                
        }else{
        	
        	return null;
        	
        }
    	
    }
    
    
    //词典分词
    public function nextToken11(){
    	
    	$first=array_sum(explode(' ',microtime()));
    	
        if ($this->_input === null||$this->_input=="") { 
    		
            return null;
             
        }
        
    	$counts=strlen($this->_input)/3;
        
        $this->_position++;
    	
    	if($this->_position>$counts){
    		
    		return null; 
    		
    	}
    	
        
        $currentChar="";

		$i=$this->_position-1;	
        
        for($j=2;;$j++){
        	
        	if($i+$j>$counts){
        		
        		$char=substr($this->_input,$i*3,($j-1)*3);
        		
        		$char=$char.'哈';
        		
        	}else{
        	
        		$char=substr($this->_input,$i*3,$j*3);
        		
        	}

        	$currentChar=$char;
        	
        	if(!in_array($char,$this->_thisAllChineseArray)){
			
        		$token = new Zend_Search_Lucene_Analysis_Token( 
                                      substr($currentChar,0,-3), 
                                      $i, 
                                      $i+$j-1);
                                      
                $this->_position=$i+$j-1;
             
                $token = $this->normalize($token);

                $last=array_sum(explode(' ',microtime()));
                
                $this->_sumTime+=$last-$first;
                
                //echo $this->_sumTime.'<br />';
                
                var_dump($token);
                
                return $token;
        		
        	}
        	
        }

    	
    }
    
    
    //字典多词分词
    public function nextToken(){
    	
    	if ($this->_input === null||$this->_input=="") { 
    		
            return null;
             
        }
        
    	$counts=strlen($this->_input)/3;
        
        $this->_position++;
    	
    	if($this->_position>$counts){
    		
    		return null; 
    		
    	}
    	
    	$valueArray=array();
    	
    	$char=substr($this->_input,($this->_position-1)*3,3);
    	
    	foreach($this->_thisAllChineseArray as $value){
    		
    		if(strpos($value,$char)==0){
    			
    			$valueArray[]=$value;
    			
    		}
    		
    	}

    	if(count($valueArray)==0){

    		$token = new Zend_Search_Lucene_Analysis_Token( 
                                      $char,
                                      $this->_position-1, 
                                      $this->_position);
                                      
            $token = $this->normalize($token);
            
            var_dump($token);echo "aa";
                
            return $token;
    		
    	}else{
    		
    		$condidateArray=array();
    		
    		for($i=2;;$i++){
    			
    			if($this->_position+$i-1>$counts){
    				
    				break;
    				
    			}
    			
    			$nextChar=substr($this->_input,($this->_position-1)*3,$i*3);
    			
    			if(in_array($nextChar,$valueArray)){
    				
    				$condidateArray[]=$nextChar;
    				
    			}
    			
    		}
    		
    		if(count($condidateArray)>0){
    			
    			usort(&$condidateArray,array("Zend_Search_Lucene_Analysis_Analyzer_Common_Myanlalysis","compare_length"));
    			
    			$length=strlen($condidateArray[0])/3;
    			
    			$token = new Zend_Search_Lucene_Analysis_Token( 
                                      $condidateArray[0],
                                      $this->_position-1, 
                                      $this->_position+$length-1);
                                      
                $this->_position=$this->_position+$length-1;
                                      
            	$token = $this->normalize($token);
            	
            	var_dump($token);echo "bb";
            	
            	return $token;
    			
    		}else{
    			
    			$token = new Zend_Search_Lucene_Analysis_Token( 
                                      $char,
                                      $this->_position-1, 
                                      $this->_position);
                                      
            	$token = $this->normalize($token);
                
            	var_dump($token);echo "cc";
            	
            	return $token;
    			
    		}
    		
    	}
    	
    }
    
	public function compare_length($a,$b){
		
		$a=strlen($a);
		
		$b=strlen($b);
		
		if($a==$b){
			
			return 0;
			
		}else if($a<$b){
			
			return 1;
		}else{
			
			return -1;
			
		}
		
	}
    
    public function getChineseWords2(){
    	
    	$first=array_sum(explode(' ',microtime()));
    	
    	require_once MY_ROOT.'/config/data/sDict.php';
    	
    	$this->_thisAllChineseArray=Zend_Registry::get('allChineseArray');
    	
    	$last=array_sum(explode(' ',microtime()));
                
        $sumTime=$last-$first;
        
        echo $sumTime;
    	
    	
    	
    }
    
    public function getChineseWords3(){
    	
    	$first=array_sum(explode(' ',microtime()));
    	
    	$fileName=MY_ROOT.'/config/data/aa.php';
		
    	$gz = gzopen($fileName,'r');
		
		while (!gzeof($gz)) {
   			$buffer .= gzgets($gz, 4096);
		}
		
		gzclose($gz);
		
		eval('$this->_thisAllChineseArray='.$buffer.';');
		
		$last=array_sum(explode(' ',microtime()));
                
        $sumTime=$last-$first;
        
        echo $sumTime;
    	
    }
    
    public function getChineseWords(){
    	
    	$first=array_sum(explode(' ',microtime()));
    	
    	$contents=file_get_contents(MY_ROOT."/config/data/ddd2.php");

		eval('$this->_thisAllChineseArray='.gzuncompress($contents).';');
		
		$last=array_sum(explode(' ',microtime()));
                
        $sumTime=$last-$first;
        
        echo $sumTime;
    	
    }
     
} 